# encoding=utf-8
# ---------------------------------------
#   版本：0.1
#   日期：2016-11-10
#   作者：九茶<bone_ace@163.com>
# ---------------------------------------

from scrapyWebSite.scrapy_redis.spiders import RedisSpider
from scrapy.http import Request
from scrapy.linkextractors import LinkExtractor
import urlparse
import re
import os
from scrapyWebSite.items import WebMgrItem


class WebSiteSpider(RedisSpider):
    name = "default"
    redis_key = "default:start_urls"
    start_urls = [

    ]
    nodeId = ''
    siteId = ''
    headers = {
        "Accept": "*/*",
        "Accept-Encoding": "gzip,deflate",
        "Accept-Language": "en-US,en;q=0.8,zh-TW;q=0.6,zh;q=0.4",
        "Connection": "keep-alive",
        "Content-Type": " application/x-www-form-urlencoded; charset=UTF-8",
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) "
                      "Chrome/38.0.2125.111 Safari/537.36",
    }

    def __init__(self, spider=None, node_id=None, site_id=None, start_url=None, *args, **kwargs):
        if spider:
            self.name = spider
            self.redis_key = spider + ":start_urls"
        if node_id:
            self.nodeId = node_id
        if site_id:
            self.siteId = site_id
        if start_url:
            self.start_urls = [start_url]
        super(WebSiteSpider, self).__init__(*args, **kwargs)

    def parse(self, response):
        print 'We Get: %s' % response.url
        item = WebMgrItem()
        item['title'] = response.xpath("/html/head/title/text()").extract()
        item['html'] = response.body
        item['url'] = response.url
        item['site_id'] = self.siteId
        item['node_id'] = self.nodeId
        parse = urlparse.urlparse(response.url)
        hostname = parse.hostname
        le = LinkExtractor(deny='javascript:;', allow_domains=hostname)
        links = le.extract_links(response)
        if links:
            for link in links:
                yield Request(url=link.url, callback=self.parse)
        yield item
